package org.ykx.demo.file

import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.types.StructType
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
import org.apache.spark.sql.types.LongType
import org.apache.spark.sql.types.StringType
import org.apache.spark.sql.types.IntegerType
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.types.DateType
import org.apache.spark.sql.types.DoubleType

object CsvConvertParquet  {
  val conf = new SparkConf().setAppName("CsvToParquet")
                            .setMaster("local[*]")
  val sc = new SparkContext(conf)
  val sqlContext = new SQLContext(sc)
  val schema= StructType(Array(
          StructField("id", StringType,false),
          StructField("date",DateType,false),
          StructField("value",DoubleType,true)))
  
   def main(args: Array[String]): Unit = {
     val csvFilePath = "hdfs://master:8020/test/eight_effective_power.csv"
//     val csvFilePath = "hdfs://master:8020/tmp/df_tem.csv"
     val tableName = "eight_effective_power.parquet"
     val df = sqlContext.load("com.databricks.spark.csv", Map("path" -> csvFilePath, "header" -> "false"))
     df.write.parquet("hdfs://master:8020/data/parquet/" + tableName)
     println("=====================转换成功=======================")
     
     val filePath = "hdfs://master:8020/data/parquet/eight_effective_power.parquet"
     val ndf = sqlContext.read.parquet(filePath)
     ndf.show(100)
      
     sc.stop()
   }       
  
  
  
}